##================================================##
##                                                ##
##       Computational Social Science             ##
##       Doctoral Seminar (16:194:630)            ##
##       Spring 2026, Rutgers University          ## 
##       Katya Ognyanova, katya@ognyanova.net     ## 
##                                                ##
##================================================##



# ================  Getting started: LLMs in R ================

# Packages we would need today:
# (remove # and run this if they are not already installed)

# install.packages("ellmer")       # LLM API calls
# install.packages("guardianapi")  # Guardian newspaper API
# install.packages("tidyverse")    # Data processing


# ------~~ Your API keys --------

# To call an LLM from R, you need an API key from your chosen provider.
# Today we will use OpenAI (gpt-4o-mini) as the default, with Gemini
# as a free alternative. The code is nearly identical for both.
# You should have received your OpenAI API keys by email. 

# You should have received OpenAI API keys from me over email.

# To get your Google Gemini API Key:
#  1. Go to https://aistudio.google.com
#  2. Sign in with your Google account
#  3. Go to "Projects" and create a new project. Give it any name you like.
#  4. Go to "API Keys" and click "Create API key" on the right. In the project dropdown, select the one you just created.
#  5. Copy and save the generated API key.key.
# The free tier will give you  more than enough free requests for class.

# NEVER paste your API key directly into a script you might share.
# You will forget to delete it (and believe me I've done that!)

# Sys.setenv() sets an environment variable for the current R session.
# Packages can read these variables and use them to authenticate you
# Run these lines in your console (replacing with real key):
# Sys.setenv(OPENAI_API_KEY = "your_openai_key")   # OpenAI
# Sys.setenv(GOOGLE_API_KEY = "your_gemini_key")   # Gemini

# Check if your keys are available:
nzchar(Sys.getenv("OPENAI_API_KEY"))   # OpenAI
nzchar(Sys.getenv("GOOGLE_API_KEY"))   # Gemini



# ================  ~~ Your first LLM call  ================

library(ellmer)
library(tidyverse)

# chat_openai() creates a connection to the OpenAI API.
# The system prompt defines the model's role for the conversation.
# We'll use gpt-4o-mini -- it's fast, cheap, and good enough for many CSS tasks.
# For Gemini: swap chat_openai() for chat_google_gemini(model = "gemini-2.5-flash-lite")

my_llm <- chat_openai(model = "gpt-4o-mini",
                      system_prompt = "You are a helpful and very funny research assistant" )

# The $chat() method sends a message and returns the response as a string:
my_llm$chat("What are three ways LLMs can help social scientists?")

# As you will see, my_llm stores our conversation history.
# This is useful for multi-turn conversations, as it allows following up:

my_llm$chat("What was the first one again?")

# On the other hand, it can be problematic for annotation:
# if we reuse the same my_llm object across documents,
# earlier texts can influence how later texts get coded.
# So, we always create a new my_llm for each document we annotate.

# See the conversation history so far:
my_llm$get_turns()

# See the system prompt:
my_llm$get_system_prompt()

# See which model you are using:
my_llm$get_model()

# Get approximate cost for the conversation so far:
my_llm$get_cost()

# ellmer supports many LLM providers beyond OpenAI and Gemini.
# They work the same way, just swap chat_openai() for another function.
# (and, of course, you are going to need the corresponding API keys)
#
# chat_anthropic()   Anthropic's Claude models  
# chat_huggingface() Numerous open-source models with the Hugging Face API 
# chat_ollama()      Run models locally on your own machine (e.g. for sensitive data)
#                    You need to install Ollama and download a model (e.g. Llama, Mistral)

# Next week we will talk more about model parameters (temperature, seed, reasoning, etc.)
# and their relationship to reproducibility.


# ================  Getting data: The Guardian  ================

# To make our LLM tasks more interesting & realistic, we'll use recent news.
# The Guardian provides a free API returning article text and other info:
#    1. Go to https://open-platform.theguardian.com/access/
#    2. Click "Register for a developer key"
#    3. Fill in the short form and copy the API key from the confirmation email
# Store the key in your console (not in a saved script):
# Sys.setenv(GU_API_KEY = "your-gu-key")

# NOTE: If you do NOT want to get a Guardian API key, skip this entire section. 
# At the start of the next section, you'll load the pre-saved data from Posit Cloud.

library(guardianapi)
 
# Verify your key is loaded:
nzchar(Sys.getenv("GU_API_KEY"))
 
# gu_content() retrieves recent Guardian articles.
# 'from_date' gives us starting data (today-3 days)
# 'show_fields' specifies which data fields to include (all) 
# 'tag' specifies to only download news tagged as 'us-news'
guardian <- gu_content(tag         = "us-news/us-news",
                       from_date   = Sys.Date() - 3,
                       show_fields = "all")

# What sections did we get, and how many articles each?
count(guardian, section_name)

# We will keep up to 3 articles per section in our data:
guardian <- group_by(guardian, section_name)
guardian <- slice_head(guardian, n = 3)
guardian <- arrange(guardian, desc(web_publication_date))
guardian <- ungroup(guardian)

count(guardian, section_name)
dim(guardian)

# Save the data for future use:
saveRDS(guardian, "guardian_data.rds")

# Check out the first article: 
guardian$body_text[1]
guardian$web_title[1]
guardian$section_name[1]



# ================  Text Annotation with LLMs  ================

# We now use the LLM to annotate our Guardian articles. 
# If you didn't get data through the API, get it here:

guardian <- readRDS("guardian_data.rds")


# ------~~ Simple classification --------

# Classify a single article's section using only the text.
# The true section label comes directly from the Guardian's section_name.

my_article  <- guardian$body_text[1]
my_title    <- guardian$web_title[1]
true_label  <- guardian$section_name[1]

# Preview the article and its true label:
my_article
true_label

# Get a list of the sections that are actually present in our data.
# We pass these to the LLM as the allowed options to choose from.
# Remember that paste() combines multiple strings into one text,
# and 'collapse' parameter says what to use in between them
section_options <- paste(unique(guardian$section_name), collapse = ", ")
section_options

# Write our classification prompt:
classify_prompt <- paste("What section does the following news article belong to?
                          Reply with only the section name and nothing else. 
                          Choose exactly one of: ", section_options,
                          ". Article title: ", my_title,
                          ". Article text: ", my_article)
classify_prompt

# Remember to use a fresh chat for each document:
my_llm <- chat_openai(model = "gpt-4o-mini",
                      system_prompt = "You are a news classification assistant. 
                                       Follow the instructions precisely.")

predicted <- my_llm$chat(classify_prompt)

predicted
true_label


# ------~~ Entity extraction --------

# Extract named entities (people, organizations, locations) from the same article.
# Note we create a new my_llm -- no memory of the classification task above.

my_llm <- chat_openai(model = "gpt-4o-mini",
                      system_prompt = "You are a research assistant specializing in text analysis.")

entity_prompt <- paste("Extract all named entities from the following news article.
                        Include the names of people, organizations, locations, and events.
                        Do not include dates or years as separate entities.
                        Return them as a comma-separated list with no other text.
                        Article: ", my_article)
entity_prompt

entities_list <- my_llm$chat(entity_prompt)

# Split our entity list into individual entities:
entities <- strsplit(entities_list, ", ")
entities


# ------~~ Summarization --------

my_llm <- chat_openai(model = "gpt-4o-mini", 
                      system_prompt = "You are a research assistant specializing in text analysis.")

summary_prompt <- paste("Summarize the following news article in exactly one sentence 
                         of no more than 30 words. Article:", my_article)

summary_text <- my_llm$chat(summary_prompt)


# ------~~ Structured output: movies! --------

# So far, we've made three separate API calls per article -- 
# one for classification, one for entities, one for summarization.
# That's pretty inefficient: instead, we could  ask for all 3 at once. 

# ellmer's 'type' system defines the exact structure we want back.
# chat_structured() returns a nice R list (no manual JSON parsing needed!)
# Some basic types we can use: type_integer() is integer numbers, 
# type_number() is floating point numbers, and type_string() is text.
# type_array() is a vector of elements from the same type.

# Let's ask an important question: 
# Which Guardian article would make the best movie?

movie_type <- type_object("Movie plot evaluation",
                           title  = type_string("The article headline" ),
                           rating = type_integer("Movie potential score from 1 to 10" ),
                           plot   = type_string("A 3-sentence movie plot based on the article" ) )

movie_prompt <- paste("Read this news article and assess its potential as a movie plot.
                       Score it from 1 (deadly boring) to 10 (instant blockbuster),
                       then write a 3-sentence movie plot inspired by the story. 
                        Article:", my_article)
movie_prompt

my_llm <- chat_openai(model = "gpt-4o-mini",
                      system_prompt = "You are a text analysis and movie expert.")
                              
result <- my_llm$chat_structured(movie_prompt, type = movie_type)

result$title
result$rating
result$plot


# ================  Parallel and batch processing  ================

# We can use a loop to sequentially submit multiple prompts to an LLM.
# While that might sometimes be useful, we have better options.
# Parallel processing can send multiple prompts at a time, saving us time.
# Batch processing runs over multiple prompts slowly, but costs less.
# Note that the batch option is only available for Open AI and Claude.

# ------~~ Parallel process: movie plots --------

# Instead of calling the API once per article in a loop, we can send all
# requests at the same time with parallel_chat_structured().
# Each article still gets its own independent call -- no context leakage --
# but they all run concurrently, so 10 articles take about as long as 1.

movie_df <- guardian[1:10, c("web_title", "body_text")]
movie_df

movie_prompt <- "Read this news article and assess its potential as a movie plot.
                 Score it from 1 (deadly boring) to 10 (instant blockbuster),
                 then write a 3-sentence movie plot inspired by the story. "
movie_df$prompt <- paste(movie_prompt, movie_df$body_text)

# Inspect the first prompt before sending:
movie_df$prompt[1]

my_llm <- chat_openai(model = "gpt-4o-mini",
                      system_prompt = "You are a text analysis and movie expert.")

# Below, as.list() is needed because parallel_chat_structured() requires a list:
movie_results <- parallel_chat_structured(my_llm, as.list(movie_df$prompt), 
                                          type = movie_type)

# Which articles have the most movie potential?
arrange(movie_results, desc(rating))


# ------~~ Parallel process: annotate news --------

# Now let's roll our sleeves and get on to a real research task:
# section classification, entity extraction, and summarization,
# all in a single structured call.

ann_type <- type_object("News article annotation",
                         section    = type_string("The news section"),
                         confidence = type_integer("Confidence in classification (1-100)"),
                         summary    = type_string("A one-sentence summary of the article"),
                         entities   = type_array(description = "Named entities mentioned in the article",
                                                 items =  type_string() ) )

ann_prompt <- paste("Annotate the following news article from the Guardian.
                     Classify it into one of these sections: ", section_options, 
                     ". Assess your confidence in the classification, from 1 to 100.
                     Summarize the article in one sentence of no more than 30 words.
                     Extract all named entities (people, organizations, locations).
                        Article title: ", my_title,
                     ". Article text: ", my_article ) 
ann_prompt 

# Try it on one article first:
my_llm     <- chat_openai(model = "gpt-4o-mini")
annotation <- my_llm$chat_structured(ann_prompt, type = ann_type)
annotation

guardian$section_name[1]
annotation$section
annotation$confidence

# Now we'll annotate our articles using parallel processing.

ann_prompt_par <- paste("Annotate the following news article from the Guardian.
                        Classify it into one of these sections: ", section_options, 
                        ". Assess your confidence in the classification, from 1 to 100.
                        Summarize the article in one sentence of no more than 30 words.
                        Extract all named entities (people, organizations, locations)." ) 
ann_prompt_par 

# First, create a data frame to store our prompts and results.
ann_df        <- guardian[, c("web_title", "section_name", "body_text")]
ann_df$prompt <- paste(ann_prompt_par, "Title:",   ann_df$web_title, 
                                       "Article:", ann_df$body_text)
# Check what the prompt looks like:
ann_df$prompt[1]

my_llm      <- chat_openai(model = "gpt-4.1")
annotations <- parallel_chat_structured(my_llm, as.list(ann_df$prompt), 
                                        type = ann_type)

ann_df$llm_section     <- annotations$section
ann_df$llm_summary     <- annotations$summary
ann_df$llm_confidence  <- annotations$confidence
 
# Confusion matrix (table of predicted vs. actual values)
# This is a hard task - many articles fit well in multuple categories!
table(Predicted = ann_df$llm_section,
      True      = ann_df$section_name)

ann_df$correct <- (ann_df$llm_section == ann_df$section_name)
number_correct <- sum(ann_df$correct)
number_total   <- nrow(ann_df)

# Overall accuracy (percent correctly classified):
number_correct/number_total 

ann_df <- group_by(ann_df, correct)
summarize(ann_df, mean_confidence=mean(llm_confidence))


# ------~~ Batch processing --------

# parallel_chat_structured() is fast and great for a smaller number of documents.
# For a large N of documents, batch_chat_structured() is more cost-efficient:
# it uses the OpenAI Batch API which costs 50% less, BUT you may have to wait
# up to 24 hours for results. It also requires a file path to save progress 
# so you don't lose your work if R closes before results arrive.

my_llm  <- chat_openai(model = "gpt-4o-mini")

batch_result <- batch_chat_structured(my_llm, as.list(ann_df$prompt), type = ann_type,
                                          path="batch_results.json")

# Results arrive as a data frame with one row per article:
batch_result$section
batch_result$summary

# If you close R before the batch completes, you can resume by running the same
# call again with the same path -- ellmer picks up where it left off:
# 
# batch_result <- batch_chat_structured(my_llm, as.list(ann_df$prompt), type = ann_type,
#                                       path="batch_results.json")



# ================  Content Types  ================

# Beyond text, ellmer supports several other content types
# that can be mixed into any prompt:

# Images (PNG, JPEG, WebP, GIF):
#   content_image_url("https://...")   image from a web URL
#   content_image_file("path_to_img")  image from a local file
#   content_image_plot()               your current R graphics window

# PDFs:
#   content_pdf_url("https://...")     PDF from a web URL
#   content_pdf_file("path/to/file")   PDF from a local file

# Audio and video (Gemini only, via google_upload()):
#   google_upload("recording.mp3")
#   google_upload("interview.mp4")

# You can mix and match in a single call:
# my_llm$chat(content_image_url("..."), content_pdf_file("..."), "Compare these.")

# Not all providers support all types. OpenAI and Gemini both handle
# images and PDFs. Audio and video require Gemini.


# ================  Image Annotation  ================

# The Guardian returns a thumbnail image for each article in the 'thumbnail' column.
# We can use these images for annotation -- and since we know each article's
# section, we have ground truth context for the images too.

# Let's look at the first article and its image:
guardian$web_title[1]
guardian$section_name[1]
guardian$thumbnail[1]

# Open the link to the thumbnail so we can see it:
browseURL(guardian$thumbnail[1])

my_llm <- chat_openai(model = "gpt-4o-mini")

my_llm$chat(content_image_url(guardian$thumbnail[1]),
                        "Describe this image in 1 sentence")

# ------~~ Can you see a human? --------

# Starting simple: a binary question about image content.

human_prompt <- "Is there a human being clearly visible in this image? 
                 Answer only Yes or No."

my_llm <- chat_openai(model = "gpt-4o-mini")

my_llm$chat(content_image_url(guardian$thumbnail[1]),
                        human_prompt)


# Ask the same question across several images: 
# (note that we could also do this with parallel_chat)
for (i in 1:5) {
  
  my_llm <- chat_openai(model = "gpt-4o-mini")
  
  my_llm$chat(content_image_url(guardian$thumbnail[i]), 
                          human_prompt)
  
  cat( guardian$web_title[i], "\n")
  cat("  Human present:", response, "\n")
}


# ------~~ Identify the public figure --------

# Let's try identifying a few well-known public figures: 

image_person_prompt <-  "Look at this image carefully. If a well-known public figure
                         or government official is visible, return their full name.
                         If you cannot recognize the person, say only: Unidentified.
                         If no person is visible, say only: No person visible."
 
my_llm <- chat_openai(model = "gpt-4o-mini")
my_llm$chat(content_image_file("DT.jpg"),
                        image_person_prompt)

my_llm <- chat_openai(model = "gpt-4o-mini")
my_llm$chat(content_image_file("BO.jpg"),
                        image_person_prompt)

my_llm <- chat_openai(model = "gpt-4o-mini")
my_llm$chat(content_image_file("KO.jpg"),
                        image_person_prompt)

# As you can see, Open AI's policy generally bans identifying people.
# Let's see if Gemini will be more responsive to our request.
# Gemini does not allow image URLs, only local image files.
my_llm <- chat_google_gemini(model = "gemini-2.5-flash-lite")
my_llm$chat(content_image_file("DT.jpg"),
                        image_person_prompt)

my_llm <- chat_google_gemini(model = "gemini-2.5-flash-lite")
my_llm$chat(content_image_file("JB.jpg"),
                        image_person_prompt)

my_llm <- chat_google_gemini(model = "gemini-2.5-flash-lite")
my_llm$chat(content_image_file("BO.jpg"),
                        image_person_prompt)

# ------~~ Guess the emotion! --------

image_em_prompt <- "If a person is visible in this image, describe the primary emotion
                    expressed on their face. Reply with only a single word. For example: 
                    sad, neutral, serious, happy, angry, concerned, or surprised.
                    If no person is visible, say: NA"

my_llm <- chat_openai(model = "gpt-4o-mini")
my_llm$chat(  content_image_url(guardian$thumbnail[1]), 
                          image_em_prompt)

my_llm <- chat_openai(model = "gpt-4o-mini")
my_llm$chat(content_image_file("DT.jpg"),
                        image_em_prompt)

my_llm <- chat_openai(model = "gpt-4o-mini")
my_llm$chat(content_image_file("JB.jpg"),
                        image_em_prompt)

my_llm <- chat_openai(model = "gpt-4o-mini")
my_llm$chat(content_image_file("BO.jpg"),
                        image_em_prompt)

# ------~~ What breed is this puppy? --------

# Now let's do something important: look at puppy pictures!

dog_url <- "https://upload.wikimedia.org/wikipedia/commons/6/68/Szczenie_Jack_Russell_Terrier3.jpg"

my_llm <- chat_openai(model = "gpt-4o-mini")
my_llm$chat(content_image_url(dog_url),
                        "What breed or breeds of dog are visible in this image?
                         Name each breed you can identify. Respond with only breed names.")

# ------~~ Identify meme politics --------

# Now we ask the model to do something genuinely hard: decode political irony.
# Meme classification requires cultural knowledge, sarcasm detection, and
# an understanding of the political environment. Complicated for machine coding!
 

meme_type <- type_object("Political meme evaluation",
                          view       = type_string("Political leaning" ),
                          reason     = type_string("Two-sentence politics reasoning" ),
                          confidence = type_integer("Confidence in the political leaning, 1-100" ),
                          rating     = type_integer("Persuasiveness score from 1-10" ), 
                          title      = type_string("An appropriate title for the meme" ) )

meme_prompt <- "What political viewpoint does this meme express?
                Reply with one word: liberal, conservative, or neither.
                Evaluate your confidence in the politicla leaning, 1 to 100.
                Then, explain your reasoning in 2 sentences. 
                Score the meme's persuasiveness from 1 (least) to 10 (most).
                Finally, give the meme an appropriate short title."
 
  
my_llm <- chat_openai(model = "gpt-4o-mini")
my_llm$chat_structured(content_image_file("meme_1.jpg"),
                                   meme_prompt, type = meme_type)

my_llm <- chat_openai(model = "gpt-4o-mini")
my_llm$chat_structured(content_image_file("meme_2.jpg"),
                                   meme_prompt, type = meme_type)


my_llm <- chat_openai(model = "gpt-4o-mini")
my_llm$chat_structured(content_image_file("meme_3.jpg"),
                                   meme_prompt, type = meme_type)

 
# ================ THE END ================